# -*- coding: utf-8 -*-
'''
Created on Apr 11, 2013

@author: LONG HOANG GIANG
'''
import pyLib
import os
import re
import logging
import json
from urlparse import urljoin

logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', datefmt='%d/%m/%Y %H:%M:%S')


def cindycandy218():
    logging.info("crawling cindycandy218")
    url = 'http://cindycandy218.wordpress.com/2013/04/08/bo-roi-ma-vuong-tong-tai-chuong-240-mot-nua-ke-hoach/'
    global CHAPTERS
    while url != '':
        tree = pyLib.loadWeb(url).build_tree()
        contentNode = tree.xpath("//div[@class='entry']/div[@class='entry-inner']")[0]
        prevNode = tree.xpath("//div[@class='nav-previous']/a")
        newUrl = ''
        if len(prevNode) > 0:
            href = prevNode[0].get('href')
            if href != '':
                newUrl = urljoin(url, href)
        pyLib.Etree.cleanNodeNextSibling(contentNode.xpath(".//div[@class='wpadvert']"), True)
        pyLib.Etree.cleanNodeNextSibling(contentNode.xpath(".//div[@id='jp-post-flair']"), True)
        name = pyLib.stringify(contentNode.xpath("//h2[@class='post-title']")[0])
        name = re.sub(r"\s*\-\s*", " - ", name).replace("–  ", "- ")
        path = pyLib.crc32unsigned(url)
        filepath = OUTPUT + "/files/" + path
        detail = pyLib.cleanHTML(pyLib.Etree.tostring(contentNode))
        print detail
        detail = re.sub(r"\n", "<br />", detail)
        detail = """<b>{0}</b><br /><hr /><br />{1}""".format(name, detail)
        pyLib.gzip(filepath, json.dumps(detail))
        CHAPTERS.append({'name': name, 'path': path})
        url = newUrl
    logging.info("finished cindycandy218")
        

def hachinh12():
    url = 'http://hachinh12.wordpress.com/2013/01/11/bo-roi-ma-vuong-tong-tai-chuong-196/'
    repeatCount = 0
    while url != '' and repeatCount < 2:
        tree = pyLib.loadWeb(url).build_tree()
        contentNode = tree.xpath("//div[@class='entry']")[0]
        name = pyLib.stringify(tree.xpath("//h2[@class='post-title']")[0])
        name = re.sub(r"\s*\-\s*", " - ", name).replace("–  ", "- ")
        path = pyLib.crc32unsigned(url)
        filepath = OUTPUT + "/files/" + path
        newUrl = ''
        prevNode = tree.xpath("//div[@class='nav-previous']/a")
        if len(prevNode) > 0:
            href = prevNode[0].get('href')
            if href == '': break
            newUrl = urljoin(url, href)
        if url in ['http://hachinh12.wordpress.com/2012/08/17/bo-roi-ma-vuong-tong-tai-155/', 
                   'http://hachinh12.wordpress.com/2012/08/17/bo-roi-ma-vuong-tong-tai/']:
            repeatCount += 1 
            continue
        pyLib.Etree.cleanNodeNextSibling(contentNode.xpath(".//div[@class='wpadvert']"), True)
        pyLib.Etree.cleanNodeNextSibling(contentNode.xpath(".//div[@id='jp-post-flair']"), True)
        pyLib.Etree.clean(contentNode.xpath(".//div[@id='pd_rating_holder_6129339_post_192']"))
        detail = pyLib.cleanHTML(pyLib.Etree.tostring(contentNode))
        print name
        print detail
        detail = re.sub(r"\n", "<br />", detail)
        detail = """<b>{0}</b><br /><hr /><br />{1}""".format(name, detail)
        pyLib.gzip(filepath, json.dumps(detail))
        CHAPTERS.append({'name': name, 'path': path})
        url = newUrl
        
def matsuri2111():
    
    url = 'http://matsuri2111.wordpress.com/category/bo%CC%89-r%C6%A1i-ma-v%C6%B0%C6%A1ng-to%CC%89ng-ta%CC%80i/'
    
    def getDetail(url):
        tree = pyLib.loadWeb(url).build_tree()
        contentNode = tree.xpath("//div[contains(@id, 'post-')]//div[@class='entry clearfix']")[0]
        pyLib.Etree.cleanNodeNextSibling(contentNode.xpath(".//div[@class='wpadvert']"), True)
        pyLib.Etree.cleanNodeNextSibling(contentNode.xpath(".//div[@id='jp-post-flair']"), True)
        detail = pyLib.cleanHTML(pyLib.Etree.tostring(contentNode))
        print detail
        detail = re.sub(r"\n", "<br />", detail)
        return detail
        
    
    while True:
        tree = pyLib.loadWeb(url).build_tree()
        for node in tree.xpath("//div[contains(@id, 'post-')]//h1[@class='title']/a"):
            name = pyLib.stringify(node)
            name = re.sub(r"\s*\-\s*", " - ", name).replace("–  ", "- ")
            href = node.get('href')
            if href == '': continue
            print name, href
            detail = getDetail(href)
            path = pyLib.crc32unsigned(href)
            filepath = OUTPUT + "/files/" + path
            detail = """<b>{0}</b><br /><hr /><br />{1}""".format(name, detail)
            pyLib.gzip(filepath, json.dumps(detail))
            CHAPTERS.append({'name': name, 'path': path})
            
        prevNode = tree.xpath("//div[@class='nav-prev fl']/a")
        if len(prevNode) == 0: break
        href = prevNode[0].get('href')
        if href == '': break
        url = urljoin(url, href)

def getListChapter(urls):
    data = []
    for url in urls:
        while True:
            tree = pyLib.loadWeb(url).build_tree()
            for node in tree.xpath("//h2[@class='post-title']/a"):
                name = pyLib.stringify(node)
                name = re.sub(r"\s*\-\s*", " - ", name).replace("–  ", "- ")
                href = node.get('href')
                if href == '': continue
                href = urljoin(url, href)
                print name, href
                data.append({'name': name, 'url': href, 'path': pyLib.crc32unsigned(href)})
            prevNode = tree.xpath("//div[@class='nav-previous']/a")
            if len(prevNode) == 0: break
            href = prevNode[0].get('href')
            if href == '': break
            url = urljoin(url, href)
    return data
    
    
    
if __name__ == '__main__':
    
    OUTPUT = "/longhoanggiang/truyentext/bo-roi-ma-vuong-tong-tai"
    pyLib.createIfNotExistsPath(OUTPUT + "/files/")

    CHAPTERS = []

    cindycandy218()
    hachinh12()
    matsuri2111()
    
    CHAPTERS.reverse()
    pyLib.gzip(OUTPUT + "/data", pyLib.encryptCipher(json.dumps(CHAPTERS)))
    
    logging.info("finished")
    os._exit(1)